{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as Data\n",
    "torch.manual_seed(8) \n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import gc\n",
    "import sys\n",
    "sys.setrecursionlimit(50000)\n",
    "import pickle\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.set_default_tensor_type('torch.cuda.FloatTensor')\n",
    "from tensorboardX import SummaryWriter\n",
    "torch.nn.Module.dump_patches = True\n",
    "import copy\n",
    "import pandas as pd\n",
    "#then import my own modules\n",
    "from AttentiveFP import Fingerprint, Fingerprint_viz, save_smiles_dicts, get_smiles_dicts, get_smiles_array, moltosvg_highlight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import f1_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from rdkit.Chem import rdMolDescriptors, MolSurf\n",
    "# from rdkit.Chem.Draw import SimilarityMaps\n",
    "from rdkit import Chem\n",
    "# from rdkit.Chem import AllChem\n",
    "from rdkit.Chem import QED\n",
    "%matplotlib inline\n",
    "from numpy.polynomial.polynomial import polyfit\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib\n",
    "from IPython.display import SVG, display\n",
    "import seaborn as sns; sns.set(color_codes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of all smiles:  93087\n",
      "number of successfully processed smiles:  93087\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAC/CAYAAAB+KF5fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE8FJREFUeJzt3XtMk9fjBvCnFZBbpXxdp5mw6Vba37yBRhQigwxFt8VLcJuLbt7mglvG1ClOEgdTZxaDDJ0ShmKMWXTLLtkkyOJtJpqJQyPBuOlsETVEQ0URKqKV0v7+MHRWKC2HlrYvzychG+ec9/QcwKfnfd/2VGa1Wq0gIqIekXt7AERE/ojhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJCDA2wPwVXfv3ofF4v0NpwYPDsedOy3eHoZH9Yc5Av1jnv44R7lchsjIsB4fx/B0wGKx+kR4AvCZcXhSf5gj0D/m2R/mCPC0nYhICMOTiEgAw5OISADDk4hIAG8Ykd8wWwBTm9lh/cDAAARwOUB9hOFJfsPUZsbZSwaH9fEvD0HAQP5JU9/g8zQRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQDuHEv0BO5WT65ieBI9gbvVk6v4HEpEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDvM6M+4+h949bGVrSazHzfOPkVhif1GUfvG1eEB+Ney0O+b5z8ikvP8/X19di0aRPmzZuHcePGQavVorKyssu2ZWVlmDVrFsaMGYPk5GTk5+fDZDJ1anf79m2sXbsWkyZNQlxcHObPn4+qqqo+65OIqDdcCs/r16+jvLwcoaGhSEhIcNiutLQUWVlZGD9+PEpKSrBs2TLs378f2dnZdu1MJhMWL16Ms2fPIicnB4WFhQgLC8PixYtx8eJFj/dJRNRbLp0jxcfH4/Tp0wCAY8eO4fjx453atLe3Y8uWLUhNTcX69esBAAkJCQgMDEROTg4WL16M2NhYAMAvv/wCvV6PX3/9FaNGjQIATJw4Ea+//joKCgqwe/duj/VJROQOLq085XLnzaqrq9HQ0ID09HS78pkzZyIwMBCHDx+2lR07dgwajcYWcgAQFBSEGTNmoKKiAi0tLR7rk4jIHdx2b1Ov1wMAYmJi7MpDQkIQHR1tq+9oq9FoOvWh1WrR3t6O2tpaj/VJROQObru12dTUBACIiIjoVBcREWGr72jrqB0A3L1712N9umrw4PAetfcklUrh7SG4hbWxFYrw4C7rFOHBCA0dCNX/QoWOB+D0+N6O0R2PIZXfZXf6wxwBD7xUSSaTuVTuqF1P2vamT2fu3GmBxWLt0TGeoFIp0NBwz9vDcItWkxn3Wh52Ku94qVJrqwkN7e09Pt5W7+T43ozRHY8hpd+lI/44R7lcJrRYcttpu1KpBAC71WCH5uZmu1WhUql02O7JvjzRJxGRO7gtPNVqNQDYXYcEgAcPHqCurs7uuqVarYZOp+vUx+XLlzFgwAC8+OKLHuuTiMgd3BaecXFxUKlUKC0ttSs/ePAg2traMG3aNFtZWloadDodLl26ZCt79OgRysvLkZiYiPDwcI/1SUTkDgPWd7yA0olDhw6hpqYG58+fR1VVFaKiotDY2IgbN25g+PDhkMvliIyMxK5du3D37l0EBwfj5MmTyMvLQ2pqKpYsWWLrS6vV4siRIygrK4NKpcKtW7ewefNmXL58Gfn5+Xj22WcBwCN9uurBg0ewev+SJ8LCBqK19ZG3h+EWbe0W3Lx9v1P5wKAAPHpkxjBVOIK6eXO7o+M7ODu+N2N0x2NI6XfpiD/OUSaTITQ0qOfHWa2uRYRWq+2yfNiwYXYvmi8tLcXu3btx9epVREZGYubMmVi+fDmCg+3vYDY0NCAvLw8nTpyAyWTCyJEjsXr1akyYMKHTY3iiT2d4w8j97pucv7c9rJv3tjs6voOz43szRnc8hpR+l4744xxFbxi5HJ79DcPT/Rie0vldOuKPc/T63XYiov6E4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIAfVUjkRo4+Xhl4vFdouwX8eGWJYHgSuZGjj1cGHr+T6v+iI/jxyhLB50AiIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAdwbiyRDJpfhvqnrvTQBYGBgAPfSJLdheJJkmNracV7X4LA+/uUh3EuT3IbPw0REAhieREQCeA5D/Yaza6IAYLH20WDI7zE8qd9wdk0UAGI1qj4aDfk7hif5DGcrQ64KyZcwPMlnOFsZclVIvoQ3jIiIBDA8iYgEMDyJiAQwPImIBDA8iYgEMDyJiAS4NTwrKyuh1Wq7/Lpy5Ypd21OnTmHu3LkYO3YsEhMTkZubC6PR2KnP+/fvY9OmTUhKSsLYsWMxZ84c/PHHH10+vqt9EhH1lkde55mVlYX4+Hi7sqioKNv/V1ZWIiMjA1OmTMHKlStx69Yt5OfnQ6fT4fvvv4dc/l+mZ2Zm4uLFi8jKykJUVBR+++03ZGZmori4GCkpKUJ9EhH1lkfCc8SIEYiLi3NYv2XLFsTExGDbtm22UFOpVHj//fdx6NAhvPHGGwCAEydOoKKiAoWFhUhLSwMAJCQkoK6uDps3b7YLT1f7JCJyhz5fjhkMBly4cAGzZ8+2Ww1OnjwZQ4YMweHDh21lR48ehUKhwJQpU2xlMpkM6enpqK2tRU1NTY/7JDFmC3DfZHb4ZbZ4e4REfcsjK8/c3FwsX74cISEhmDBhAj755BOMHj0aAKDT6QAAMTExnY7TaDTQ6/W27/V6PdRqdadTbq1Wa+tLrVb3qE8SY2oz4+wlg8N6bjRM/Y1b/9oVCgUWLVqEiRMnQqlU4sqVK9i1axfmzZuHffv2ITY2Fk1NTQCAiIiITsdHRETg4sWLtu+bmpowfPjwLtt11D/5X1f6dNXgweE9PsZTVCqFt4cAa2MrFOHBDutDQwdC9b9Q4T4U4cEIDAzo9jE8Xe9Km6CBgbAOcHzCNiAQvf45+Ttf+HvtC24Nz5EjR2LkyJG27ydMmIDU1FTMmDEDW7duxd69e211Mpmsyz6eLnfUridtu+vDkTt3WmDxgW18VCoFGhrueXsYaDWZca/loeP6VhMa2tuF+lCEB+Ney0O0tXX/GJ6ud6VNS6vJ6eYljo5XhAe79HPyZ77y99oTcrlMaLHk8fMslUqFpKQkHD9+HACgVCoB/LdafFJzc7Pd6lGpVDpsB/y30uxJn0S+zmx5fJnEEX6QnW/ok4tUFst/dxM6rkvq9XokJSXZtdPpdBg3bpzte7VajSNHjsBisdhd9+y4xqnRaHrcJ5Gv4/Vl/+Dx56+GhgZUVFTYXro0dOhQjB49GmVlZXahevr0aRgMBkybNs1WlpaWBqPRaFu1djhw4ABGjBgBtVrd4z6JiNzBrU9fq1evRnR0NEaNGoVBgwahtrYWJSUlePjwIVatWmVrl5WVhaVLl2LVqlV45513YDAYkJ+fj9jYWLz22mu2dikpKZg0aRLWrVuHpqYmREVF4cCBAzh37hyKiorsHtvVPomI3MGt4anValFeXo59+/bhwYMHUCqVmDhxIj766CPbKTYAJCYmori4GDt27EBGRgbCwsIwdepUrFmzBgMGDLC1k8lkKCoqQkFBAbZu3Qqj0Qi1Wo3CwkKkpqbaPbarfRIRuYNbwzMjIwMZGRkutU1OTkZycrLTduHh4cjNzUVubq7b+iQi6i3esyMiEsDwJCISwNc7kFs4+9hggB8dTNLC8CS3cPaxwQA/OpikhaftREQCGJ5ERAIYnkREAhieREQCGJ5ERAIYnkREAvhSJaI+xNfDSgfDk6gP8fWw0sHTdiIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAQxPIiIBDE8iIgEMTyIiAdwMmQAAZgtganO8wzl3N/cdznajHxgYgAAuizyO4UkAHgfn2UsGh/Xc3dx3ONuNPv7lIQgYyH/ansbnJyIiAQxPIiIBDE8iIgG8MEIkMbyh1DcYnkQSwxtKfYM/wX6CL0Uici+GZz/BlyJRB57WuwfDUwKcrSoBrizpPzytdw9J/YTu37+PrVu34tChQzAajVCr1fj4448xZcoUbw/No5ytKgGuLIncTVKL88zMTJSVlWHFihXYuXMn1Go1MjMzceLECW8PjYgkRjIrzxMnTqCiogKFhYVIS0sDACQkJKCurg6bN29GSkqKl0dIRFIimZXn0aNHoVAo7E7RZTIZ0tPTUVtbi5qaGi+OjoikRjIrT71eD7VaDbnc/vlAq9UCAHQ6HdRqtcv9yeUyt46vN6yQ4ZG53WG9XC5HaHBgt30EDOi+jafru2sTMjAA7eZAnx6jO+pDBgZ4fQyu1vfm79+X/u24QnS8MqvVKon7sNOnT8fw4cOxc+dOu/Jr165h+vTp+OKLLzB//nwvjY6IpEYyp+3A49N0kToiop6STHgqlUo0NTV1Km9ubgYARERE9PWQiEjCJBOearUaV65cgcVisSvX6XQAAI1G441hEZFESSY809LSYDQacfz4cbvyAwcOYMSIET26WURE5Ixk7ranpKRg0qRJWLduHZqamhAVFYUDBw7g3LlzKCoq8vbwiEhiJHO3HQBaWlpQUFCAw4cP2709c+rUqd4eGhFJjKTCk4ior0jmmicRUV9ieBIRCWB4elF9fT02bdqEefPmYdy4cdBqtaisrOyybVlZGWbNmoUxY8YgOTkZ+fn5MJlMfTzinjt9+jSys7Mxffp0xMbGIjk5GZmZmbh8+XKntqdOncLcuXMxduxYJCYmIjc3F0aj0Quj7pmqqiosXboUr7zyCsaMGYOEhAQsXLiwy928/HWOjuzYsQNarRazZ8/uVCe1uT6N4elF169fR3l5OUJDQ5GQkOCwXWlpKbKysjB+/HiUlJRg2bJl2L9/P7Kzs/twtGJ++OEH3Lx5E4sXL0ZJSQmys7Nx8+ZNvPXWW6iurra1q6ysREZGBoYOHYri4mKsXbsWx48fR0ZGRqfX7voao9GIESNGIDs7G7t378aXX36JoKAgZGRkoLy83NbOn+fYFb1ej5KSEjzzzDOd6qQ21y5ZyWva29tt/3/06FGrRqOx/vXXX3ZtzGazdfLkydYPP/zQrvzHH3+0ajQaa3V1dZ+MVdTt27c7lTU3N1snTJhgzczMtJW9+eab1tmzZ9v9TP7880+rRqOxlpeX98lY3amtrc2anJxsXbBgga1MSnNsb2+3vv3229aNGzda33vvPeusWbPs6qU0V0e48vSip3eA6kp1dTUaGhqQnp5uVz5z5kwEBgbi8OHDnhqeWwwePLhT2aBBg/DCCy+gvr4eAGAwGHDhwgXMnj3b7mcyefJkDBkyxOfn2JWAgAAoFAoEBj7evUhqc9y7dy/q6+vx6aefdqqT2lwdYXj6OL1eDwCIiYmxKw8JCUF0dLSt3p80NjZCr9fb5tTxFtqn5wg8flutv8zRYrHAbDbDYDBg+/btuHbtGhYtWgRAOnMEgLq6Omzfvh25ubkIDw/vVC+luXZHMu8wkqqOzU662tgkIiKiy81QfJnVakVOTg4sFguWLl0KwPkcL1682KdjFLVy5Urbqio8PBzbtm1DcnIyAOnM0Wq14vPPP0dSUpLDN59IZa7OcOXpJxxtqedvW+3l5eXh2LFj2LBhA1566SW7On+f45o1a/Dzzz/j22+/RUpKClauXImDBw/atfH3Of7000/4+++/kZOT47Stv8/VGYanj1MqlQDgcLs9f9pqb+vWrdizZw/WrVuHOXPm2MqlMsfo6GiMHTsWqampKCgoQFJSEjZu3AiLxSKJOTY2NmLLli1YtmwZQkJCYDQaYTQaYTabYbFYYDQaYTKZJDFXVzA8fVzHblBPXyd68OAB6urquryu5Iu++eYbFBcXY82aNVi4cKFdXcccuroWptPp/GaOTxszZgyam5vR2NgoiTkaDAbcu3cPX3/9NeLj421fVVVV0Ol0iI+Px44dOyQxV1fwmqePi4uLg0qlQmlpKaZNm2YrP3jwINra2uzKfFVhYSGKioqwYsUKfPDBB53qhw4ditGjR6OsrAyLFi2y3aE9ffo0DAaDX8zxaVarFWfOnMGgQYOgVCoREBDg93N8/vnn8d1333Uq/+qrr9Da2opNmzbhueeek+TvsysD1q9fv97bg+jPDh06hJqaGpw/fx5VVVWIiopCY2Mjbty4geHDh0MulyMyMhK7du3C3bt3ERwcjJMnTyIvLw+pqalYsmSJt6fQrT179qCgoACvvvoq0tPTUV9fb/tqbGyESqUC8Pgf5p49e1BTU4OIiAicO3cOGzZsQExMDLKzs116WZe3rF69Gv/88w+MRiMaGxtRXV2NvLw8VFRU4LPPPkNcXBwA/54jAAQFBSEqKqrT1++//w6LxYKsrCzbKbm/z9UV3FXJyzo+3fNpw4YNs9vYubS0FLt378bVq1cRGRmJmTNnYvny5QgODu6roQpZsGABzpw502Xd03M8efIkduzYgX///RdhYWGYOnUq1qxZ4/PXyPbt24eysjJcu3YN9+7dg0KhwOjRo/Huu+8iNTXVrq2/zrE7CxYsgNFoRGlpqV25FOf6JIYnEZEA/187ExF5AcOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISMD/A1RRVNImzYfZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "task_name = 'muv'\n",
    "tasks = [\n",
    "    \"MUV-466\",\"MUV-548\",\"MUV-600\",\"MUV-644\",\"MUV-652\",\"MUV-689\",\"MUV-692\",\"MUV-712\",\"MUV-713\",\"MUV-733\",\"MUV-737\",\"MUV-810\",\"MUV-832\",\"MUV-846\",\"MUV-852\",\"MUV-858\",\"MUV-859\"\n",
    "]\n",
    "raw_filename = \"../data/muv.csv\"\n",
    "feature_filename = raw_filename.replace('.csv','.pickle')\n",
    "filename = raw_filename.replace('.csv','')\n",
    "prefix_filename = raw_filename.split('/')[-1].replace('.csv','')\n",
    "smiles_tasks_df = pd.read_csv(raw_filename)\n",
    "smilesList = smiles_tasks_df.smiles.values\n",
    "print(\"number of all smiles: \",len(smilesList))\n",
    "atom_num_dist = []\n",
    "remained_smiles = []\n",
    "canonical_smiles_list = []\n",
    "for smiles in smilesList:\n",
    "    try:        \n",
    "        mol = Chem.MolFromSmiles(smiles)\n",
    "        atom_num_dist.append(len(mol.GetAtoms()))\n",
    "        remained_smiles.append(smiles)\n",
    "        canonical_smiles_list.append(Chem.MolToSmiles(Chem.MolFromSmiles(smiles), isomericSmiles=True))\n",
    "    except:\n",
    "        print(\"not successfully processed smiles: \", smiles)\n",
    "        pass\n",
    "print(\"number of successfully processed smiles: \", len(remained_smiles))\n",
    "smiles_tasks_df = smiles_tasks_df[smiles_tasks_df[\"smiles\"].isin(remained_smiles)]\n",
    "# print(smiles_tasks_df)\n",
    "smiles_tasks_df['cano_smiles'] =canonical_smiles_list\n",
    "assert canonical_smiles_list[8]==Chem.MolToSmiles(Chem.MolFromSmiles(smiles_tasks_df['cano_smiles'][8]), isomericSmiles=True)\n",
    "\n",
    "plt.figure(figsize=(5, 3))\n",
    "sns.set(font_scale=1.5)\n",
    "ax = sns.distplot(atom_num_dist, bins=28, kde=False)\n",
    "plt.tight_layout()\n",
    "# plt.savefig(\"atom_num_dist_\"+prefix_filename+\".png\",dpi=200)\n",
    "plt.show()\n",
    "plt.close()\n",
    "\n",
    "# print(len([i for i in atom_num_dist if i<51]),len([i for i in atom_num_dist if i>50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed = 68\n",
    "start_time = str(time.ctime()).replace(':','-').replace(' ','_')\n",
    "start = time.time()\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 800\n",
    "p_dropout = 0.2\n",
    "fingerprint_dim = 250\n",
    "\n",
    "radius = 3\n",
    "T = 2\n",
    "weight_decay = 3.5 # also known as l2_regularization_lambda\n",
    "learning_rate = 3.7\n",
    "per_task_output_units_num = 2 # for classification model with 2 classes\n",
    "output_units_num = len(tasks) * per_task_output_units_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature dicts file saved as ../data/muv.pickle\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MUV-466</th>\n",
       "      <th>MUV-548</th>\n",
       "      <th>MUV-600</th>\n",
       "      <th>MUV-644</th>\n",
       "      <th>MUV-652</th>\n",
       "      <th>MUV-689</th>\n",
       "      <th>MUV-692</th>\n",
       "      <th>MUV-712</th>\n",
       "      <th>MUV-713</th>\n",
       "      <th>MUV-733</th>\n",
       "      <th>MUV-737</th>\n",
       "      <th>MUV-810</th>\n",
       "      <th>MUV-832</th>\n",
       "      <th>MUV-846</th>\n",
       "      <th>MUV-852</th>\n",
       "      <th>MUV-858</th>\n",
       "      <th>MUV-859</th>\n",
       "      <th>mol_id</th>\n",
       "      <th>smiles</th>\n",
       "      <th>cano_smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [MUV-466, MUV-548, MUV-600, MUV-644, MUV-652, MUV-689, MUV-692, MUV-712, MUV-713, MUV-733, MUV-737, MUV-810, MUV-832, MUV-846, MUV-852, MUV-858, MUV-859, mol_id, smiles, cano_smiles]\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smilesList = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())<151]\n",
    "# uncovered = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())>150]\n",
    "\n",
    "# smiles_tasks_df = smiles_tasks_df[~smiles_tasks_df[\"cano_smiles\"].isin(uncovered)]\n",
    "\n",
    "if os.path.isfile(feature_filename):\n",
    "    feature_dicts = pickle.load(open(feature_filename, \"rb\" ))\n",
    "else:\n",
    "    feature_dicts = save_smiles_dicts(smilesList,filename)\n",
    "# feature_dicts = get_smiles_dicts(smilesList)\n",
    "\n",
    "remained_df = smiles_tasks_df[smiles_tasks_df[\"smiles\"].isin(feature_dicts['smiles_to_atom_mask'].keys())]\n",
    "uncovered_df = smiles_tasks_df.drop(remained_df.index)\n",
    "uncovered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = []\n",
    "for i,task in enumerate(tasks):    \n",
    "    negative_df = remained_df[remained_df[task] == 0][[\"smiles\",task]]\n",
    "    positive_df = remained_df[remained_df[task] == 1][[\"smiles\",task]]\n",
    "    negative_test = negative_df.sample(frac=1/10,random_state=random_seed)\n",
    "    negative_valid = negative_df.drop(negative_test.index).sample(frac=1/9,random_state=random_seed)\n",
    "    negative_train = negative_df.drop(negative_test.index).drop(negative_valid.index)\n",
    "    \n",
    "    positive_test = positive_df.sample(frac=1/10,random_state=random_seed)\n",
    "    positive_valid = positive_df.drop(positive_test.index).sample(frac=1/9,random_state=random_seed)\n",
    "    positive_train = positive_df.drop(positive_test.index).drop(positive_valid.index)\n",
    "    \n",
    "    weights.append([(positive_test.shape[0]+negative_test.shape[0])/negative_test.shape[0],\\\n",
    "                    (positive_test.shape[0]+negative_test.shape[0])/positive_test.shape[0]])\n",
    "    train_df_new = pd.concat([negative_train,positive_train])\n",
    "    valid_df_new = pd.concat([negative_valid,positive_valid])\n",
    "    test_df_new = pd.concat([negative_test,positive_test])\n",
    "    if i==0:\n",
    "        train_df = train_df_new\n",
    "        test_df = test_df_new\n",
    "        valid_df = valid_df_new\n",
    "    else:\n",
    "        train_df = pd.merge(train_df, train_df_new, on='smiles', how='outer') \n",
    "        test_df = pd.merge(test_df, test_df_new, on='smiles', how='outer')\n",
    "        valid_df = pd.merge(valid_df, valid_df_new, on='smiles', how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1790038\n",
      "atom_fc.weight torch.Size([250, 39])\n",
      "atom_fc.bias torch.Size([250])\n",
      "neighbor_fc.weight torch.Size([250, 49])\n",
      "neighbor_fc.bias torch.Size([250])\n",
      "GRUCell.0.weight_ih torch.Size([750, 250])\n",
      "GRUCell.0.weight_hh torch.Size([750, 250])\n",
      "GRUCell.0.bias_ih torch.Size([750])\n",
      "GRUCell.0.bias_hh torch.Size([750])\n",
      "GRUCell.1.weight_ih torch.Size([750, 250])\n",
      "GRUCell.1.weight_hh torch.Size([750, 250])\n",
      "GRUCell.1.bias_ih torch.Size([750])\n",
      "GRUCell.1.bias_hh torch.Size([750])\n",
      "GRUCell.2.weight_ih torch.Size([750, 250])\n",
      "GRUCell.2.weight_hh torch.Size([750, 250])\n",
      "GRUCell.2.bias_ih torch.Size([750])\n",
      "GRUCell.2.bias_hh torch.Size([750])\n",
      "align.0.weight torch.Size([1, 500])\n",
      "align.0.bias torch.Size([1])\n",
      "align.1.weight torch.Size([1, 500])\n",
      "align.1.bias torch.Size([1])\n",
      "align.2.weight torch.Size([1, 500])\n",
      "align.2.bias torch.Size([1])\n",
      "attend.0.weight torch.Size([250, 250])\n",
      "attend.0.bias torch.Size([250])\n",
      "attend.1.weight torch.Size([250, 250])\n",
      "attend.1.bias torch.Size([250])\n",
      "attend.2.weight torch.Size([250, 250])\n",
      "attend.2.bias torch.Size([250])\n",
      "mol_GRUCell.weight_ih torch.Size([750, 250])\n",
      "mol_GRUCell.weight_hh torch.Size([750, 250])\n",
      "mol_GRUCell.bias_ih torch.Size([750])\n",
      "mol_GRUCell.bias_hh torch.Size([750])\n",
      "mol_align.weight torch.Size([1, 500])\n",
      "mol_align.bias torch.Size([1])\n",
      "mol_attend.weight torch.Size([250, 250])\n",
      "mol_attend.bias torch.Size([250])\n",
      "output.weight torch.Size([34, 250])\n",
      "output.bias torch.Size([34])\n"
     ]
    }
   ],
   "source": [
    "x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array([smilesList[0]],feature_dicts)\n",
    "num_atom_features = x_atom.shape[-1]\n",
    "num_bond_features = x_bonds.shape[-1]\n",
    "\n",
    "loss_function = [nn.CrossEntropyLoss(torch.Tensor(weight),reduction='mean') for weight in weights]\n",
    "model = Fingerprint(radius, T, num_atom_features,num_bond_features,\n",
    "            fingerprint_dim, output_units_num, p_dropout)\n",
    "model.cuda()\n",
    "# tensorboard = SummaryWriter(log_dir=\"runs/\"+start_time+\"_\"+prefix_filename+\"_\"+str(fingerprint_dim)+\"_\"+str(p_dropout))\n",
    "\n",
    "# optimizer = optim.Adam(model.parameters(), learning_rate, weight_decay=weight_decay)\n",
    "optimizer = optim.Adam(model.parameters(), 10**-learning_rate, weight_decay=10**-weight_decay)\n",
    "model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "params = sum([np.prod(p.size()) for p in model_parameters])\n",
    "print(params)\n",
    "for name, param in model.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, param.data.shape)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, dataset, optimizer, loss_function):\n",
    "    model.train()\n",
    "    np.random.seed(epoch)\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    #shuffle them\n",
    "    np.random.shuffle(valList)\n",
    "    batch_list = []\n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, train_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[train_batch,:]\n",
    "        smiles_list = batch_df.smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "#         print(torch.Tensor(x_atom).size(),torch.Tensor(x_bonds).size(),torch.cuda.LongTensor(x_atom_index).size(),torch.cuda.LongTensor(x_bond_index).size(),torch.Tensor(x_mask).size())\n",
    "        \n",
    "        model.zero_grad()\n",
    "        # Step 4. Compute your loss function. (Again, Torch wants the target wrapped in a variable)\n",
    "        loss = 0.0\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where(y_val != -1)[0]\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "\n",
    "            loss += loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "        # Step 5. Do the backward pass and update the gradient\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "def eval(model, dataset):\n",
    "    model.eval()\n",
    "    y_val_list = {}\n",
    "    y_pred_list = {}\n",
    "    losses_list = []\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    batch_list = []\n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, eval_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[eval_batch,:]\n",
    "        smiles_list = batch_df.smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "        atom_pred = atoms_prediction.data[:,:,1].unsqueeze(2).cpu().numpy()\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where((y_val=='0') | (y_val=='1'))[0]\n",
    "#             print(validInds)\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "#             print(validInds)\n",
    "            loss = loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "#             print(y_pred_adjust)\n",
    "            y_pred_adjust = F.softmax(y_pred_adjust,dim=-1).data.cpu().numpy()[:,1]\n",
    "            losses_list.append(loss.cpu().detach().numpy())\n",
    "            try:\n",
    "                y_val_list[i].extend(y_val_adjust)\n",
    "                y_pred_list[i].extend(y_pred_adjust)\n",
    "            except:\n",
    "                y_val_list[i] = []\n",
    "                y_pred_list[i] = []\n",
    "                y_val_list[i].extend(y_val_adjust)\n",
    "                y_pred_list[i].extend(y_pred_adjust)\n",
    "#             print(y_val,y_pred,validInds,y_val_adjust,y_pred_adjust)            \n",
    "    eval_roc = [roc_auc_score(y_val_list[i], y_pred_list[i]) for i in range(len(tasks))]\n",
    "    eval_prc = [auc(precision_recall_curve(y_val_list[i], y_pred_list[i])[1],precision_recall_curve(y_val_list[i], y_pred_list[i])[0]) for i in range(len(tasks))]\n",
    "#     eval_precision = [precision_score(y_val_list[i],\n",
    "#                                      (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "#     eval_recall = [recall_score(y_val_list[i],\n",
    "#                                (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "    eval_loss = np.array(losses_list).mean()\n",
    "    \n",
    "    return eval_roc, eval_prc, eval_loss # eval_precision, eval_recall, \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:\t0\n",
      "train_roc:[0.42612659306044387, 0.617240512691631, 0.6308665731133413, 0.711857601713062, 0.36540520394479625, 0.5517446500630282, 0.5980674107651957, 0.43376758810842414, 0.667715234826656, 0.5768877361124035, 0.6747729715704808, 0.5262479733448362, 0.567303996925442, 0.516052930324961, 0.6243133348695802, 0.4565880843911717, 0.522737306843267]\n",
      "valid_roc:[0.36506864731037586, 0.6603217765692273, 0.7942176870748299, 0.6790038839387708, 0.38780542479264735, 0.5465568519789521, 0.5067305498516997, 0.5048678720445062, 0.8570785505289218, 0.6375426621160409, 0.6066393815370623, 0.5740360483686973, 0.8711293260473589, 0.600817438692098, 0.4249886000911993, 0.11796610169491524, 0.6599864130434783]\n",
      "train_roc_mean:0.5569232766275718\n",
      "valid_roc_mean:0.576162189157693\n",
      "train_prc_mean:0.002543257957516508\n",
      "valid_prc_mean:0.005038454245508154\n",
      "\n",
      "EPOCH:\t1\n",
      "train_roc:[0.6145557109107564, 0.7389973833212602, 0.4969772353574871, 0.7559885795860101, 0.6491338329490605, 0.6098667084368263, 0.6085250598699966, 0.7327318549183501, 0.6429900710261163, 0.6022378338140876, 0.7292190222024538, 0.6402259441610019, 0.6467852364278192, 0.7966475095785441, 0.76555078162748, 0.5494622348032494, 0.37360969604347094]\n",
      "valid_roc:[0.6644159351789332, 0.7668252889191026, 0.37709750566893424, 0.8684030157642221, 0.9338713292983636, 0.4410889956531686, 0.6812685375313712, 0.48400556328233657, 0.6452847175331984, 0.6982935153583618, 0.46680309231468853, 0.6698608259183207, 0.8638433515482696, 0.9396003633060854, 0.7970816233470133, 0.9134463276836158, 0.4714673913043478]\n",
      "train_roc_mean:0.6443238055902336\n",
      "valid_roc_mean:0.6872151399770784\n",
      "train_prc_mean:0.00367497553123715\n",
      "valid_prc_mean:0.010838087742469311\n",
      "\n",
      "EPOCH:\t2\n",
      "train_roc:[0.6476845378718481, 0.7499519536389575, 0.5707674207631683, 0.7548179871520342, 0.6908947408806732, 0.6571192015932332, 0.6124558102406203, 0.7667151220205882, 0.6619010039091894, 0.6399344002977575, 0.7491641647207087, 0.6679334810870309, 0.6931455860164546, 0.8288846317581949, 0.797639135638199, 0.5756925708051367, 0.4694769910001698]\n",
      "valid_roc:[0.7490434391177133, 0.7917516428733288, 0.3984126984126984, 0.8686314827507425, 0.9242322349249047, 0.4959963395104095, 0.7227926078028747, 0.5368567454798331, 0.6659914472203465, 0.7513083048919227, 0.49386084583901774, 0.7077344284736482, 0.8570127504553733, 0.9666212534059946, 0.8073415412676698, 0.9337853107344634, 0.452445652173913]\n",
      "train_roc_mean:0.678481102317292\n",
      "valid_roc_mean:0.7131658073726383\n",
      "train_prc_mean:0.004469893906929456\n",
      "valid_prc_mean:0.01314440182857744\n",
      "\n",
      "EPOCH:\t3\n",
      "train_roc:[0.6560917988525142, 0.729210709164289, 0.5494450586834495, 0.74197715917202, 0.7128293572349886, 0.6740062804418686, 0.6006671228190216, 0.7848424279292446, 0.6693926991759502, 0.6654957972767594, 0.7795062826642944, 0.691844293385295, 0.7001060437839838, 0.8480204342273308, 0.8287927330573045, 0.5838567237235909, 0.4552640516216675]\n",
      "valid_roc:[0.773126266036462, 0.7901654203489689, 0.3875283446712018, 0.8366461046378798, 0.904954046177987, 0.4905056051246854, 0.758612822267853, 0.5519239684747335, 0.6295295971190638, 0.7460750853242321, 0.543656207366985, 0.7741273100616017, 0.869535519125683, 0.98047229791099, 0.8098495212038304, 0.9383050847457628, 0.47554347826086957]\n",
      "train_roc_mean:0.6865499396007984\n",
      "valid_roc_mean:0.7212092164034581\n",
      "train_prc_mean:0.005137971912915001\n",
      "valid_prc_mean:0.016547026828154184\n",
      "\n",
      "EPOCH:\t4\n",
      "train_roc:[0.6716166047924401, 0.7884814393211419, 0.6011368146510178, 0.7772912205567452, 0.7224465336875221, 0.6929855967538618, 0.6104637073782644, 0.7984783965491361, 0.6768899003432012, 0.6853501752427035, 0.7895288928425812, 0.7148626336848682, 0.704031087197882, 0.8694054207464169, 0.8373151857991333, 0.6222724726141508, 0.45722533537103077]\n",
      "valid_roc:[0.7503938780103534, 0.8493088601858148, 0.29795918367346935, 0.8608636052090473, 0.8755884330867518, 0.6156485929993136, 0.6331279945242985, 0.6163653222067687, 0.587440918298447, 0.7690557451649602, 0.5677580718508413, 0.7736710015970796, 0.8827413479052824, 0.9866030881017257, 0.8335613315093479, 0.9527683615819209, 0.5855978260869565]\n",
      "train_roc_mean:0.7070459657371823\n",
      "valid_roc_mean:0.7316737389407283\n",
      "train_prc_mean:0.0061103202938284345\n",
      "valid_prc_mean:0.023503688253688516\n",
      "\n",
      "EPOCH:\t5\n",
      "train_roc:[0.684116002121402, 0.8151730408172316, 0.6012962805465782, 0.7939739471805853, 0.7382315649469996, 0.6944885766072187, 0.6067752879461741, 0.7998017017309622, 0.6893167177491878, 0.6836907974318414, 0.7615626969124133, 0.7291273111306131, 0.7223716799043471, 0.8998829289059174, 0.8610762151835691, 0.6588194966606219, 0.5999511801664119]\n",
      "valid_roc:[0.7566959261760072, 0.8801268978019487, 0.3276643990929705, 0.8512679917751884, 0.9092131809011432, 0.6300617707618394, 0.7465206479580195, 0.6464997681965694, 0.5109160477155075, 0.7610921501706485, 0.5104592996816735, 0.7798311658681267, 0.9175774134790529, 0.9922797456857403, 0.8554491564067487, 0.967683615819209, 0.6195652173913043]\n",
      "train_roc_mean:0.7258620838789456\n",
      "valid_roc_mean:0.7448767291106881\n",
      "train_prc_mean:0.006170440855958711\n",
      "valid_prc_mean:0.053638810038321584\n",
      "\n",
      "EPOCH:\t6\n",
      "train_roc:[0.6999943750703116, 0.8339111216238192, 0.6561631796790838, 0.804707351891506, 0.766732315832167, 0.7193903048475762, 0.6061445147679324, 0.8092030937690101, 0.7148971314258447, 0.7154748612015756, 0.8275399384706623, 0.7458723170060539, 0.7421392376234806, 0.9069958847736626, 0.8866026894229412, 0.659634069028558, 0.6506749872643913]\n",
      "valid_roc:[0.7778528021607021, 0.8937230908678903, 0.2780045351473923, 0.8466986520447795, 0.8856758574310692, 0.6465339739190116, 0.6187542778918549, 0.6638850254983772, 0.4877335133918524, 0.7444823663253697, 0.6055025011368804, 0.814966917636322, 0.9061930783242258, 0.987511353315168, 0.8324213406292749, 0.9631638418079096, 0.6124320652173912]\n",
      "train_roc_mean:0.7497692572763869\n",
      "valid_roc_mean:0.7391491289850277\n",
      "train_prc_mean:0.010839603575543325\n",
      "valid_prc_mean:0.01889460834358468\n",
      "\n",
      "EPOCH:\t7\n",
      "train_roc:[0.7024171126432348, 0.8624987064441257, 0.6658161818903441, 0.8355496074232691, 0.7770875886537779, 0.7425429076506522, 0.6084038944007298, 0.8261809017436027, 0.7338851469157781, 0.7205382897552806, 0.8291245042440417, 0.7602671466182749, 0.7452814074643437, 0.9149106002554278, 0.8918098225635003, 0.7104987689268285, 0.6321276957038546]\n",
      "valid_roc:[0.7702003151024082, 0.9073192839338319, 0.2873015873015873, 0.8530957276673521, 0.9152656355077337, 0.6829100892244337, 0.621492128678987, 0.7222994900324524, 0.47693000225073146, 0.7369738339021616, 0.6302864938608459, 0.7707049965776865, 0.9246357012750456, 0.9915985467756585, 0.8369813041495667, 0.9719774011299435, 0.6647418478260869]\n",
      "train_roc_mean:0.7622906048998274\n",
      "valid_roc_mean:0.7508655520703831\n",
      "train_prc_mean:0.015473275939816974\n",
      "valid_prc_mean:0.04203368261415328\n",
      "\n",
      "EPOCH:\t8\n",
      "train_roc:[0.7336676148690998, 0.864228375441657, 0.6440969269150082, 0.8520164168451106, 0.8212564446409454, 0.761947384516697, 0.6078693408598472, 0.8401565845295749, 0.7763062748912585, 0.8090358549672777, 0.8224433818896177, 0.7827537223519612, 0.7617608961767302, 0.9267702568468852, 0.9105681387379485, 0.7370073864390286, 0.6388245033112583]\n",
      "valid_roc:[0.7717758271438216, 0.9163834126444597, 0.254875283446712, 0.8099154672149874, 0.9222147500560413, 0.6536261725005719, 0.6725986767054528, 0.8474733426054706, 0.3936529372045915, 0.7142207053469852, 0.6080036380172805, 0.8297969427332876, 0.9125683060109289, 0.9900090826521345, 0.8077975376196991, 0.9715254237288136, 0.7055027173913043]\n",
      "train_roc_mean:0.7818064414252885\n",
      "valid_roc_mean:0.7518788366483848\n",
      "train_prc_mean:0.018232801944542694\n",
      "valid_prc_mean:0.024175009210324342\n",
      "\n",
      "EPOCH:\t9\n",
      "train_roc:[0.7155834659209617, 0.8688703930931507, 0.6693846742643308, 0.8554389721627409, 0.8092970910964385, 0.7778759873794446, 0.6275908028281446, 0.8499213917220348, 0.7546827683667664, 0.7828463757327627, 0.8319581897031024, 0.7727505986999658, 0.7743260170239417, 0.9251702852277565, 0.9063422212641329, 0.7621411827148481, 0.651256580064527]\n",
      "valid_roc:[0.7765023632680621, 0.8959891230455472, 0.33219954648526073, 0.8403015764222069, 0.9432862586863933, 0.7256920613132006, 0.6700889801505819, 0.7994900324524803, 0.46860229574611745, 0.7638225255972696, 0.6161891768985902, 0.8156513803331051, 0.9264571948998179, 0.9936421435059037, 0.8406292749658001, 0.968813559322034, 0.647078804347826]\n",
      "train_roc_mean:0.7844374704273559\n",
      "valid_roc_mean:0.7661433116141293\n",
      "train_prc_mean:0.020439954854539307\n",
      "valid_prc_mean:0.07244491150850262\n",
      "\n",
      "EPOCH:\t10\n",
      "train_roc:[0.743973289619594, 0.889057256478867, 0.691904802404037, 0.8782441113490363, 0.8322913507528948, 0.7844771644028732, 0.6549349982894287, 0.8688940328811712, 0.7859379301484758, 0.8323446853385441, 0.890752066422032, 0.7750524327299231, 0.8202949298260598, 0.9404604796367249, 0.9122072149085314, 0.7865894112963863, 0.6669468500594328]\n",
      "valid_roc:[0.7805536799459826, 0.8601858146385678, 0.3154195011337868, 0.8357322366917981, 0.923559739968617, 0.7485701212537177, 0.542094455852156, 0.8395920259619842, 0.45824893090254326, 0.7299203640500569, 0.7669395179627103, 0.8323066392881586, 0.9323770491803279, 0.9938692098092644, 0.842453260373917, 0.9387570621468927, 0.6535326086956522]\n",
      "train_roc_mean:0.80908017685553\n",
      "valid_roc_mean:0.7643595422268313\n",
      "train_prc_mean:0.026589798790182077\n",
      "valid_prc_mean:0.06169507677971186\n",
      "\n",
      "EPOCH:\t11\n",
      "train_roc:[0.7361385661250662, 0.8874865100601689, 0.7199211884107275, 0.8781441827266239, 0.8223051261158226, 0.7710062879008258, 0.6442047268787775, 0.8762847910757879, 0.7621781342338539, 0.8385867373840762, 0.8810333963453056, 0.7913846703059692, 0.8233054345660034, 0.9419114516815666, 0.9173771807889866, 0.7773416191192297, 0.6320937340804891]\n",
      "valid_roc:[0.7674994373171281, 0.9277135735327442, 0.36621315192743764, 0.8405300434087274, 0.9446312485989689, 0.669640814458934, 0.7319187770933151, 0.8639313861845155, 0.4442943956785955, 0.714448236632537, 0.7116871305138699, 0.8838694957791468, 0.9408014571948998, 0.9936421435059037, 0.8492932056543548, 0.9769491525423728, 0.7476222826086957]\n",
      "train_roc_mean:0.8059237492823106\n",
      "valid_roc_mean:0.7867462313313027\n",
      "train_prc_mean:0.025562364938606818\n",
      "valid_prc_mean:0.07794306797695445\n",
      "\n",
      "EPOCH:\t12\n",
      "train_roc:[0.7538691480642207, 0.9045503599781205, 0.7435823836253331, 0.8865917201998572, 0.8478388757549958, 0.7986529123497952, 0.6916018075037063, 0.8810605402245274, 0.7940022390661993, 0.8866311528798735, 0.9059638978464731, 0.8081445507147214, 0.8330984712614229, 0.941769547325103, 0.9155336846878322, 0.77306603564952, 0.692142129393785]\n",
      "valid_roc:[0.784154850326356, 0.8982551552232042, 0.34807256235827666, 0.8606351382225267, 0.9258013898229097, 0.7247769389155798, 0.3648186173853525, 0.8270746407046824, 0.5185685347738014, 0.7633674630261661, 0.7867212369258753, 0.8389231120237282, 0.9423952641165756, 0.995685740236149, 0.8433652530779754, 0.9649717514124294, 0.7153532608695652]\n",
      "train_roc_mean:0.8269470268544403\n",
      "valid_roc_mean:0.7707612299659501\n",
      "train_prc_mean:0.034439085858460346\n",
      "valid_prc_mean:0.0649370203045046\n",
      "\n",
      "EPOCH:\t13\n",
      "train_roc:[0.757557494817029, 0.9109479177446299, 0.7406694732664285, 0.9018165596002855, 0.8447439865242603, 0.803534799764297, 0.692945318736458, 0.8887791620910592, 0.8084020041477783, 0.8980955925684688, 0.9077949516290448, 0.7913214535393952, 0.86350599254135, 0.9556229601248759, 0.9249221345900823, 0.788911495422177, 0.6954788588894549]\n",
      "valid_roc:[0.7715507539950484, 0.9172898255155224, 0.3791383219954648, 0.8546949965729952, 0.9437345886572517, 0.722260352322123, 0.41592516541181834, 0.8282336578581362, 0.4555480531172631, 0.7251422070534698, 0.7926330150068213, 0.8747433264887063, 0.9546903460837887, 0.9959128065395095, 0.8606931144550843, 0.9701694915254238, 0.7296195652173914]\n",
      "train_roc_mean:0.8338264797645337\n",
      "valid_roc_mean:0.7759987992832834\n",
      "train_prc_mean:0.03407065350548032\n",
      "valid_prc_mean:0.03298731323751048\n",
      "\n",
      "EPOCH:\t14\n",
      "train_roc:[0.7803665847034055, 0.9321141877208285, 0.7715562453932074, 0.9095003568879373, 0.8674276610749169, 0.815592203898051, 0.725410537119398, 0.8961738704503978, 0.8657147576486135, 0.9071485065599703, 0.9233552021942992, 0.8197392494310491, 0.8873338172915421, 0.9602348517099475, 0.929043983735607, 0.8371223848909726, 0.7117677024961793]\n",
      "valid_roc:[0.7785280216070223, 0.9220484930886019, 0.3589569160997732, 0.8163125428375599, 0.9334229993275052, 0.7757950125829329, 0.487109285877253, 0.8150208623087621, 0.48728336709430564, 0.7551763367463026, 0.8892678490222827, 0.9112480036504678, 0.955145719489982, 0.9950045413260672, 0.8841769265845874, 0.863276836158192, 0.7126358695652175]\n",
      "train_roc_mean:0.8552707119533131\n",
      "valid_roc_mean:0.7847299754921656\n",
      "train_prc_mean:0.04498250491247604\n",
      "valid_prc_mean:0.025801979020173563\n",
      "\n",
      "EPOCH:\t15\n",
      "train_roc:[0.7902986034103145, 0.9409768933962125, 0.7897831972557691, 0.9201070663811564, 0.8514855102913288, 0.8119447738817158, 0.7521631599954385, 0.9098453905527861, 0.8655605924348927, 0.9201443813777488, 0.9209644538344639, 0.8124879144416844, 0.9046531158368206, 0.9618064424577835, 0.9235506626922476, 0.8342363660489185, 0.7389370011886568]\n",
      "valid_roc:[0.7927076299797434, 0.9385905279854975, 0.43333333333333335, 0.8195110806488463, 0.9558394978704327, 0.749027682452528, 0.42573579739904177, 0.8678720445062587, 0.4307900067521945, 0.752901023890785, 0.8633469758981355, 0.8765685603467944, 0.9533242258652095, 0.9940962761126249, 0.9044687642498859, 0.9398870056497175, 0.6698369565217391]\n",
      "train_roc_mean:0.8617026779692905\n",
      "valid_roc_mean:0.786343375850751\n",
      "train_prc_mean:0.04953534273753737\n",
      "valid_prc_mean:0.02987148350140607\n",
      "\n",
      "EPOCH:\t16\n",
      "train_roc:[0.7931150860614243, 0.9474188016498382, 0.7949835572943245, 0.9194825124910777, 0.8592391761090629, 0.8264561748976257, 0.7510548523206751, 0.9119824296673171, 0.8720245196101821, 0.9201947830402283, 0.9197079209755736, 0.8369899894390813, 0.9025002135109745, 0.9650737902653612, 0.937246798040542, 0.8391790879738157, 0.7391259127186279]\n",
      "valid_roc:[0.7620976817465677, 0.9567187854067527, 0.40634920634920635, 0.8240804203792551, 0.9031607262945528, 0.7419354838709677, 0.5256673511293635, 0.8736671302735282, 0.43304073823992795, 0.7001137656427759, 0.8458390177353343, 0.9363449691991786, 0.9503642987249544, 0.9940962761126249, 0.8714090287277702, 0.9116384180790961, 0.6396059782608696]\n",
      "train_roc_mean:0.8668103297685724\n",
      "valid_roc_mean:0.7809487809513369\n",
      "train_prc_mean:0.0566305892199903\n",
      "valid_prc_mean:0.028099577204504882\n",
      "\n",
      "EPOCH:\t17\n",
      "train_roc:[0.8054738601481768, 0.9538828851470219, 0.7794480353801667, 0.9481120628122769, 0.8698758024971042, 0.8369099032573265, 0.7427621450564489, 0.9133768378141369, 0.8548277570796703, 0.9371762662448435, 0.9307053634308166, 0.8230153653929109, 0.9202927947163151, 0.9681069958847737, 0.9384658841719505, 0.8531447653589278, 0.7219349634912549]\n",
      "valid_roc:[0.7600720234076075, 0.9320190346702923, 0.33877551020408164, 0.864290610006854, 0.9291638646043487, 0.7455959734614505, 0.4170659365731234, 0.8324061196105702, 0.5471528246680171, 0.762457337883959, 0.8683492496589359, 0.915582934063427, 0.9540072859744991, 0.9959128065395095, 0.8793889648882809, 0.8992090395480224, 0.6277173913043479]\n",
      "train_roc_mean:0.8704418639931836\n",
      "valid_roc_mean:0.7805392298274898\n",
      "train_prc_mean:0.06805544783911363\n",
      "valid_prc_mean:0.0376517731354387\n",
      "\n",
      "EPOCH:\t18\n",
      "train_roc:[0.8116733362261543, 0.9630893070975562, 0.8385425525883087, 0.9420985010706637, 0.8672595797235427, 0.8385471443382787, 0.7612755160223516, 0.9292604501607717, 0.9163470185548847, 0.9348112651592693, 0.9363690277623337, 0.8424489431644082, 0.9499174424232073, 0.9689726124592025, 0.941721736157054, 0.8767526206377991, 0.7797588724741044]\n",
      "valid_roc:[0.8158901643033986, 0.9354180829367776, 0.41972789115646253, 0.8243088873657756, 0.9600986325935889, 0.79867307252345, 0.5074150125484826, 0.885025498377376, 0.4976367319378798, 0.8100113765642776, 0.8833560709413371, 0.9208304814054301, 0.980191256830601, 0.9904632152588556, 0.9217966256269949, 0.9222598870056496, 0.5040760869565217]\n",
      "train_roc_mean:0.8881674074129348\n",
      "valid_roc_mean:0.7986575867254624\n",
      "train_prc_mean:0.07305102057909654\n",
      "valid_prc_mean:0.05182436871538968\n",
      "\n",
      "EPOCH:\t19\n",
      "train_roc:[0.8311998778586697, 0.9780982511124581, 0.8692131598344389, 0.9558029978586724, 0.8801214570460799, 0.8507686455279823, 0.7826683487284752, 0.9357308199751929, 0.9206599739387377, 0.9446550975466022, 0.9412839615997628, 0.8639128947329279, 0.9497395166111537, 0.9799134383425572, 0.9462189746296283, 0.8919493711943621, 0.823403803701817]\n",
      "valid_roc:[0.7879810938555031, 0.9354180829367778, 0.5585034013605442, 0.9120402101896276, 0.9639094373458865, 0.8407687028140015, 0.38580880675336526, 0.8052851182197496, 0.6563133018230924, 0.7838452787258249, 0.8872214643019555, 0.9564225416381473, 0.9758652094717669, 0.9945504087193461, 0.9347925216598266, 0.9152542372881356, 0.44904891304347827]\n",
      "train_roc_mean:0.902667093543501\n",
      "valid_roc_mean:0.8084134547145312\n",
      "train_prc_mean:0.09037911270846676\n",
      "valid_prc_mean:0.05847144245329296\n",
      "\n",
      "EPOCH:\t20\n",
      "train_roc:[0.8375319415650161, 0.9727614091628107, 0.8754465045075693, 0.9621877230549608, 0.884626768051389, 0.8575078132575504, 0.8316049435511461, 0.9390489583415629, 0.9338888175161048, 0.9456010979808319, 0.9399866562882241, 0.8699668297907153, 0.9606606029549919, 0.9857350645664822, 0.9544292223865841, 0.9097851887891253, 0.8217609101715061]\n",
      "valid_roc:[0.8050866531622777, 0.9349648765012464, 0.5238095238095237, 0.9031299977153302, 0.9127998206680117, 0.7961564859299932, 0.4496919917864476, 0.8168752897542884, 0.6862480306099482, 0.8088737201365188, 0.8603910868576626, 0.9466119096509241, 0.9785974499089253, 0.995685740236149, 0.9384404924760602, 0.887909604519774, 0.5207201086956521]\n",
      "train_roc_mean:0.9107370854080334\n",
      "valid_roc_mean:0.809764281318749\n",
      "train_prc_mean:0.11053546156240195\n",
      "valid_prc_mean:0.05742482951329947\n",
      "\n",
      "EPOCH:\t21\n",
      "train_roc:[0.8625829677129035, 0.9815021509986251, 0.89067372569031, 0.9621163454675232, 0.8784516053596026, 0.8695241931273169, 0.8328878720492644, 0.9394597754726371, 0.9385211151283792, 0.9398049067956948, 0.948278290522258, 0.8894580463787949, 0.9789300253366356, 0.9850787569178373, 0.9587666416407116, 0.9162059356893273, 0.8308796060451689]\n",
      "valid_roc:[0.8053117263110511, 0.9333786539768865, 0.5979591836734693, 0.9193511537582819, 0.8527236045729657, 0.8592999313658202, 0.5119780971937029, 0.8548910523875753, 0.5935178933153274, 0.8222980659840728, 0.8819918144611187, 0.944558521560575, 0.9842896174863388, 0.9909173478655768, 0.9635202918376653, 0.895819209039548, 0.5027173913043479]\n",
      "train_roc_mean:0.9178307035489995\n",
      "valid_roc_mean:0.8185013856526072\n",
      "train_prc_mean:0.11569002058745904\n",
      "valid_prc_mean:0.054297941707240394\n",
      "\n",
      "EPOCH:\t22\n",
      "train_roc:[0.8583160567635761, 0.9831874695090401, 0.8883809604808075, 0.9724339757316203, 0.904924418201018, 0.8752265658215668, 0.8533363268331623, 0.9482291411551862, 0.9513168278671976, 0.9544524053224156, 0.9511546017272694, 0.8892144758958188, 0.9796951063284652, 0.9875088690222791, 0.9659547897448095, 0.9194015657481535, 0.8337238920020378]\n",
      "valid_roc:[0.8111636281791582, 0.9340584636301835, 0.5487528344671202, 0.9440255883024903, 0.9372338040798027, 0.828185769846717, 0.35957107004334926, 0.8555864626796477, 0.7134818816115237, 0.8040955631399318, 0.9038199181446113, 0.9653205566963268, 0.9749544626593807, 0.983197093551317, 0.959188326493388, 0.919322033898305, 0.47690217391304346]\n",
      "train_roc_mean:0.9244974969502602\n",
      "valid_roc_mean:0.8187564489021351\n",
      "train_prc_mean:0.1306317107138131\n",
      "valid_prc_mean:0.06077928374475879\n",
      "\n",
      "EPOCH:\t23\n",
      "train_roc:[0.8796385580894525, 0.9902502845822924, 0.907282984634575, 0.9751534618129908, 0.9282329169056954, 0.8811862725353742, 0.8506778138898392, 0.959048642328385, 0.94817112338723, 0.9497650507118266, 0.9550242781422588, 0.9051302265391423, 0.9847090557121302, 0.991077763587342, 0.9687014502664891, 0.9156530585165199, 0.8734844625573104]\n",
      "valid_roc:[0.8082376772451048, 0.9612508497620665, 0.7448979591836735, 0.9575051405071967, 0.9713068818650527, 0.8556394417753375, 0.6057494866529775, 0.8569772832637923, 0.6950258834121089, 0.8707622298065985, 0.9172351068667577, 0.9539128450832763, 0.985655737704918, 0.9947774750227066, 0.9813041495668035, 0.943954802259887, 0.5010190217391304]\n",
      "train_roc_mean:0.9331286708352267\n",
      "valid_roc_mean:0.8591301159833759\n",
      "train_prc_mean:0.14948080856325147\n",
      "valid_prc_mean:0.07861574903845782\n",
      "\n",
      "EPOCH:\t24\n",
      "train_roc:[0.8851951850601867, 0.9875818636074686, 0.9195866643987073, 0.9731620271234832, 0.9305349006310358, 0.887888145479499, 0.8703101836013228, 0.9664789021702205, 0.9616972855910584, 0.9516454204274061, 0.9620260202379628, 0.9159923544898779, 0.9882248697583056, 0.9897048389385553, 0.9666015000706177, 0.930695003464697, 0.8323781626761759]\n",
      "valid_roc:[0.8311951384199865, 0.9766598685701338, 0.6158730158730159, 0.9282613662325794, 0.9731002017484869, 0.8782887211164493, 0.39858544375998173, 0.8792304126101067, 0.6545127166329057, 0.8334470989761092, 0.8867667121418826, 0.9698836413415469, 0.9801912568306012, 0.9970481380563124, 0.9872321021431828, 0.9421468926553673, 0.5241168478260869]\n",
      "train_roc_mean:0.9364531369250929\n",
      "valid_roc_mean:0.8386199749961608\n",
      "train_prc_mean:0.18079584374144678\n",
      "valid_prc_mean:0.09480135721129555\n",
      "\n",
      "EPOCH:\t25\n",
      "train_roc:[0.9037614708387574, 0.99573865736292, 0.9271488915348416, 0.9802783725910064, 0.9383652992396145, 0.9056180864791484, 0.8803312806477364, 0.9717839733916904, 0.9701029603391634, 0.9671381160633975, 0.95501686496905, 0.9321944398994482, 0.9936195803797648, 0.9954909890733646, 0.9799594133519666, 0.9442441800462943, 0.8635973849550008]\n",
      "valid_roc:[0.8485257708755346, 0.9696351688193973, 0.6696145124716553, 0.949965729952022, 0.9704102219233355, 0.902081903454587, 0.44330367328313935, 0.9209550301344459, 0.7436416835471528, 0.8393629124004551, 0.9188267394270123, 0.9666894820898929, 0.988615664845173, 0.9918256130790192, 0.9922480620155039, 0.9251977401129944, 0.5516304347826086]\n",
      "train_roc_mean:0.9473170565390097\n",
      "valid_roc_mean:0.8583841378361136\n",
      "train_prc_mean:0.21181914695486953\n",
      "valid_prc_mean:0.09676046830931632\n",
      "\n",
      "EPOCH:\t26\n",
      "train_roc:[0.8836362759751217, 0.995768224354331, 0.9308626750581164, 0.9775910064239828, 0.9353617585694085, 0.9071322547681383, 0.8965781446002964, 0.9748927530278012, 0.9740121496870813, 0.9675839769237927, 0.9678713073130953, 0.9284349016049621, 0.9960820736185839, 0.9967397474102454, 0.9826131557233864, 0.9431457973963171, 0.9028273051451859]\n",
      "valid_roc:[0.8104884087328382, 0.9768864717878994, 0.5852607709750567, 0.9666438199680146, 0.9661510872001794, 0.859071150766415, 0.4704540269221994, 0.9007881316643487, 0.7267611973891515, 0.8341296928327646, 0.9045020463847203, 0.974674880219028, 0.9587887067395264, 0.9906902815622162, 0.9949840401276789, 0.9367231638418079, 0.5652173913043478]\n",
      "train_roc_mean:0.9506549122117559\n",
      "valid_roc_mean:0.8483656040245995\n",
      "train_prc_mean:0.22345951394023686\n",
      "valid_prc_mean:0.08658426623547269\n",
      "\n",
      "EPOCH:\t27\n",
      "train_roc:[0.9048342252864696, 0.9958458377067841, 0.9468978567783637, 0.9817094932191293, 0.9380181747096029, 0.9122043455883999, 0.9051096191127836, 0.9774998617442348, 0.9770734303595353, 0.96874709221178, 0.9686015048741614, 0.9379880706242841, 0.9973382298516811, 0.9972718887469846, 0.9915295814223167, 0.9513504946407773, 0.902627780607913]\n",
      "valid_roc:[0.7850551429214494, 0.9560389757534558, 0.7052154195011338, 0.9769248343614347, 0.9246805648957632, 0.9023106840539922, 0.4134154688569473, 0.8101529902642558, 0.7681746567634482, 0.8391353811149033, 0.9274670304683946, 0.9571070043349303, 0.97632058287796, 0.9902361489554949, 0.9956680346557228, 0.7814689265536722, 0.5251358695652174]\n",
      "train_roc_mean:0.956155734557953\n",
      "valid_roc_mean:0.837323983288128\n",
      "train_prc_mean:0.23013849948612317\n",
      "valid_prc_mean:0.09282315207519722\n",
      "\n",
      "EPOCH:\t28\n",
      "train_roc:[0.9268156469472703, 0.9968732906583091, 0.9500233883313488, 0.9860956459671663, 0.9557471033371456, 0.9221434059089858, 0.8962110845022238, 0.9867945993347922, 0.9810780553159469, 0.9766717843739339, 0.9723970495570629, 0.949244373707775, 0.9979004754177698, 0.9972612459202498, 0.9942427914099028, 0.9650765919176729, 0.9337663440312447]\n",
      "valid_roc:[0.8260184559981993, 0.9660095173351461, 0.7743764172335601, 0.9716700936714645, 0.9453037435552567, 0.8906428734843285, 0.39060004563084644, 0.8789986091794159, 0.8748593292820167, 0.8391353811149033, 0.929968167348795, 0.9758156513803331, 0.9690346083788708, 0.9947774750227065, 0.9956680346557228, 0.8779661016949153, 0.5944293478260869]\n",
      "train_roc_mean:0.9640201692140472\n",
      "valid_roc_mean:0.8644278736936805\n",
      "train_prc_mean:0.26855110921844405\n",
      "valid_prc_mean:0.09990345690068334\n",
      "\n",
      "EPOCH:\t29\n",
      "train_roc:[0.9190813686257493, 0.9970063421196576, 0.9608989624085729, 0.9868665239114919, 0.9588712241072505, 0.9246980987118381, 0.9284375356369028, 0.9828918365895859, 0.983838346761613, 0.9798974907726187, 0.9669001816227437, 0.9535468324681313, 0.9975161556637344, 0.9970980559103164, 0.9943171259301107, 0.9713830775354947, 0.9262693156732892]\n",
      "valid_roc:[0.8370470402880936, 0.9685021527305688, 0.8106575963718821, 0.9728124286040667, 0.9585294776955839, 0.9318233813772591, 0.3682409308692676, 0.8824756606397774, 0.8575286968264686, 0.8368600682593856, 0.9711232378353797, 0.9703399498060689, 0.9560564663023678, 0.986376021798365, 0.9933880528955767, 0.904632768361582, 0.4775815217391305]\n",
      "train_roc_mean:0.9664422632028883\n",
      "valid_roc_mean:0.8637632619059309\n",
      "train_prc_mean:0.2775459582029996\n",
      "valid_prc_mean:0.09039389401536263\n",
      "\n",
      "EPOCH:\t30\n",
      "train_roc:[0.931158896228083, 0.9976087695696524, 0.965697114021659, 0.9867344753747324, 0.9736039199494294, 0.9456167438668726, 0.9133845079256471, 0.9917520560607377, 0.9910510764035458, 0.9838869762104153, 0.9830646058045147, 0.9603221823915274, 0.9982314174281891, 0.9974102454945366, 0.9949118020917733, 0.9725883497722146, 0.9460604516895909]\n",
      "valid_roc:[0.8437992347512941, 0.9651031044640833, 0.7485260770975056, 0.966872286954535, 0.9641336023313158, 0.8721116449325097, 0.42847364818617384, 0.8541956420955029, 0.8825118163403105, 0.8382252559726963, 0.904274670304684, 0.9705681040383298, 0.9747267759562842, 0.9877384196185286, 0.9947560419516643, 0.944180790960452, 0.5805027173913043]\n",
      "train_roc_mean:0.9725343288401838\n",
      "valid_roc_mean:0.8659235196086573\n",
      "train_prc_mean:0.3123388026688052\n",
      "valid_prc_mean:0.18323626060311373\n",
      "\n",
      "EPOCH:\t31\n",
      "train_roc:[0.9466033460296032, 0.9981151042975621, 0.974184243352044, 0.9874660956459671, 0.974908377393789, 0.9566970246220173, 0.9254369084274148, 0.992012766932381, 0.9905518747591169, 0.9832162463943426, 0.9857704140257237, 0.970689732109655, 0.9974734534688418, 0.9976727685539946, 0.9962423900034939, 0.9798900143010896, 0.9502886737986076]\n",
      "valid_roc:[0.8413234301147874, 0.9784726943122593, 0.6845804988662132, 0.9792095042266392, 0.9798251513113652, 0.9034545870510181, 0.4519735341090577, 0.8201205377839592, 0.8624803060994823, 0.8468714448236633, 0.9345156889495225, 0.973990417522245, 0.968351548269581, 0.9906902815622163, 0.9961240310077519, 0.9028248587570621, 0.5672554347826086]\n",
      "train_roc_mean:0.9768952608303318\n",
      "valid_roc_mean:0.8636508205617314\n",
      "train_prc_mean:0.31591576689212925\n",
      "valid_prc_mean:0.10704334080723164\n",
      "\n",
      "EPOCH:\t32\n",
      "train_roc:[0.9520032785304469, 0.9982962021199533, 0.9743366218744685, 0.9903640256959314, 0.9853440369486658, 0.9578382450565762, 0.9433587068080739, 0.9953980580990227, 0.9917227962633288, 0.9890589621909991, 0.9864264798547018, 0.9751892784363891, 0.9984378113701711, 0.9973712217965092, 0.9963092910716808, 0.9696359856694238, 0.9615214807267788]\n",
      "valid_roc:[0.7958586540625703, 0.9743938363924767, 0.6775510204081633, 0.9812657071053232, 0.9652544272584622, 0.8959048272706475, 0.46155601186402007, 0.8015762633286974, 0.9329282016655414, 0.8334470989761091, 0.8885857207821737, 0.9899612137805155, 0.9640255009107469, 0.9927338782924614, 0.9945280437756497, 0.96045197740113, 0.5859375]\n",
      "train_roc_mean:0.9801536754419483\n",
      "valid_roc_mean:0.8644682284279229\n",
      "train_prc_mean:0.3250105888618875\n",
      "valid_prc_mean:0.11329619467699976\n",
      "\n",
      "EPOCH:\t33\n",
      "train_roc:[0.9700070713401797, 0.998972547048475, 0.9810660826671203, 0.9928229835831548, 0.9779338417185222, 0.9660207209827922, 0.9620217527654237, 0.9961406890667341, 0.9936131554315709, 0.9924940293415216, 0.9854924200303941, 0.9720767823409541, 0.9986691149258405, 0.9984780757769264, 0.9973908583407048, 0.9855109322246304, 0.9726524027848531]\n",
      "valid_roc:[0.8044114337159577, 0.9843643779741672, 0.673469387755102, 0.971441626684944, 0.9796009863259358, 0.905971173644475, 0.4004106776180698, 0.8354195642095502, 0.8696826468602297, 0.8769055745164961, 0.8524329240563893, 0.9835728952772075, 0.9763205828779599, 0.9947774750227065, 0.9965800273597811, 0.9471186440677967, 0.65625]\n",
      "train_roc_mean:0.9847860859041058\n",
      "valid_roc_mean:0.865219411645104\n",
      "train_prc_mean:0.41660108459941003\n",
      "valid_prc_mean:0.12705447333995804\n",
      "\n",
      "EPOCH:\t34\n",
      "train_roc:[0.9723615061954581, 0.998950371804917, 0.9885964166241424, 0.9935546038543897, 0.9827497378296313, 0.976101501488062, 0.9755031930664843, 0.9961446392314559, 0.993323178005763, 0.9929437672528768, 0.9912450424404167, 0.9784691130315786, 0.9988790673840636, 0.9989392649354336, 0.9982865893092093, 0.9876192371769354, 0.9653633893700119]\n",
      "valid_roc:[0.7729011928876885, 0.9782460910944936, 0.5786848072562358, 0.9814941740918438, 0.9836359560636628, 0.887439945092656, 0.43714350901209215, 0.8701900788131665, 0.9304523970290344, 0.8573378839590443, 0.8901773533424284, 0.9856262833675565, 0.944216757741348, 0.9940962761126249, 0.9970360237118102, 0.9349152542372882, 0.6898777173913043]\n",
      "train_roc_mean:0.9875900364118135\n",
      "valid_roc_mean:0.8654983353649575\n",
      "train_prc_mean:0.4267383809209877\n",
      "valid_prc_mean:0.12931495578108143\n",
      "\n",
      "EPOCH:\t35\n",
      "train_roc:[0.9595567555405558, 0.998813624469642, 0.9911230651471338, 0.9920556745182013, 0.9880260306858085, 0.9747961839975534, 0.9833575664271867, 0.9961327887372904, 0.9960467634481619, 0.9960337768679631, 0.9944290003335928, 0.9813473352273572, 0.9989893813875366, 0.9988186462324393, 0.9987697636905604, 0.9890382885871408, 0.9801876379690949]\n",
      "valid_roc:[0.7465676344812064, 0.9709947881259914, 0.5321995464852607, 0.9849211788896505, 0.9365613091235149, 0.8835506749027683, 0.35614875655943423, 0.8041261010662957, 0.9273013729462075, 0.8131968145620023, 0.8851750795816281, 0.9908738307095597, 0.9253187613843351, 0.9936421435059036, 0.9979480164158687, 0.9222598870056498, 0.47010869565217395]\n",
      "train_roc_mean:0.9892660166627776\n",
      "valid_roc_mean:0.8318173289057323\n",
      "train_prc_mean:0.4752882717628565\n",
      "valid_prc_mean:0.09787676656052477\n",
      "\n",
      "EPOCH:\t36\n",
      "train_roc:[0.9814819279044726, 0.9992312582233195, 0.9913108805352384, 0.9936581013561742, 0.9898128085297632, 0.9790403305809783, 0.9731654122476908, 0.998222425875159, 0.9952392313762916, 0.9948667845290159, 0.9925720004447904, 0.9844300822561692, 0.9990320835824295, 0.998673194267064, 0.9980078348584299, 0.9904610258451649, 0.9793937850229241]\n",
      "valid_roc:[0.8140895791132118, 0.9709947881259914, 0.573015873015873, 0.9885766506739777, 0.986998430845102, 0.8709677419354838, 0.45836185261236595, 0.7688919796012982, 0.9232500562682872, 0.8664391353811149, 0.8706230104592997, 0.9580196212639744, 0.9663023679417122, 0.995685740236149, 0.9952120383036935, 0.943728813559322, 0.7533967391304348]\n",
      "train_roc_mean:0.9905058333785339\n",
      "valid_roc_mean:0.8649737893216054\n",
      "train_prc_mean:0.48917847604770837\n",
      "valid_prc_mean:0.09032853098783365\n",
      "\n",
      "EPOCH:\t37\n",
      "train_roc:[0.9903572633913504, 0.9992275623493931, 0.9929409763565232, 0.9988472519628836, 0.992019789752153, 0.9821059619444009, 0.9825165355228647, 0.9986016416884584, 0.9954521261952393, 0.9953708011538104, 0.9964490900329888, 0.9949872822740187, 0.9993309989466791, 0.9990669788562508, 0.9984426918016457, 0.9923592374718033, 0.9881940906775344]\n",
      "valid_roc:[0.7978843124015305, 0.9630636755041921, 0.6122448979591837, 0.9766963673749143, 0.9567361578121498, 0.895218485472432, 0.43828428017339716, 0.805980528511822, 0.8800360117038037, 0.8293515358361774, 0.8417462482946794, 0.972393337896418, 0.9571948998178506, 0.994096276112625, 0.9929320565435477, 0.9105084745762712, 0.7846467391304348]\n",
      "train_roc_mean:0.993898251786941\n",
      "valid_roc_mean:0.8593537814777311\n",
      "train_prc_mean:0.5127712361375085\n",
      "valid_prc_mean:0.1461933244852603\n",
      "\n",
      "EPOCH:\t38\n",
      "train_roc:[0.9903130675152276, 0.9994567065328268, 0.9914986959233429, 0.9994325481798716, 0.9935690613387315, 0.9897849582671351, 0.9860053882996921, 0.9978708612149126, 0.9958081744269275, 0.9949714649049347, 0.9967122576819007, 0.9944406431748203, 0.9991993338457598, 0.9990208599404001, 0.9989816170731526, 0.992193374319961, 0.9934114450670741]\n",
      "valid_roc:[0.7344136844474455, 0.9637434851574892, 0.4750566893424035, 0.9808087731322823, 0.9661510872001793, 0.8814916495081218, 0.37942048825005703, 0.8400556328233658, 0.8935404006302048, 0.8129692832764506, 0.889267849022283, 0.9851699749030345, 0.9665300546448088, 0.991371480472298, 0.9943000455996353, 0.8673446327683616, 0.7669836956521738]\n",
      "train_roc_mean:0.9948629681003924\n",
      "valid_roc_mean:0.8463893474606232\n",
      "train_prc_mean:0.5380585424155614\n",
      "valid_prc_mean:0.11349348214880556\n",
      "\n",
      "EPOCH:\t39\n",
      "train_roc:[0.990706812593414, 0.9993495261889626, 0.9960913137154845, 0.9989900071377588, 0.9956408466915379, 0.9921046939216959, 0.9877979245067853, 0.9994667277625476, 0.998770348890561, 0.9974140070097081, 0.9970829163423404, 0.994667479807821, 0.9996370313434111, 0.9993436923513551, 0.9989964839771942, 0.9956322703348225, 0.9873238240787909]\n",
      "valid_roc:[0.766148998424488, 0.9791525039655563, 0.4587301587301587, 0.9865204477952936, 0.9762385115444967, 0.8622740791580874, 0.3709787816563997, 0.9093648585999073, 0.9099707404906595, 0.8195676905574516, 0.8933606184629377, 0.94569929272188, 0.9596994535519126, 0.995685740236149, 0.9977200182398541, 0.9077966101694915, 0.9072690217391305]\n",
      "train_roc_mean:0.9958244650973055\n",
      "valid_roc_mean:0.8615398544731679\n",
      "train_prc_mean:0.5989088820236459\n",
      "valid_prc_mean:0.10613986734233204\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_param ={}\n",
    "best_param[\"roc_epoch\"] = 0\n",
    "best_param[\"loss_epoch\"] = 0\n",
    "best_param[\"valid_roc\"] = 0\n",
    "best_param[\"valid_loss\"] = 9e8\n",
    "\n",
    "for epoch in range(epochs):    \n",
    "    train_roc, train_prc, train_loss = eval(model, train_df)\n",
    "    valid_roc, valid_prc, valid_loss = eval(model, valid_df)\n",
    "    train_roc_mean = np.array(train_roc).mean()\n",
    "    valid_roc_mean = np.array(valid_roc).mean()\n",
    "    train_prc_mean = np.array(train_prc).mean()\n",
    "    valid_prc_mean = np.array(valid_prc).mean()\n",
    "    \n",
    "#     tensorboard.add_scalars('ROC',{'train_roc':train_roc_mean,'valid_roc':valid_roc_mean},epoch)\n",
    "#     tensorboard.add_scalars('Losses',{'train_losses':train_loss,'valid_losses':valid_loss},epoch)\n",
    "\n",
    "    if valid_roc_mean > best_param[\"valid_roc\"]:\n",
    "        best_param[\"roc_epoch\"] = epoch\n",
    "        best_param[\"valid_roc\"] = valid_roc_mean\n",
    "        if valid_roc_mean > 0.75:\n",
    "             torch.save(model, 'saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(epoch)+'.pt')             \n",
    "    if valid_loss < best_param[\"valid_loss\"]:\n",
    "        best_param[\"loss_epoch\"] = epoch\n",
    "        best_param[\"valid_loss\"] = valid_loss\n",
    "\n",
    "    print(\"EPOCH:\\t\"+str(epoch)+'\\n'\\\n",
    "        +\"train_roc\"+\":\"+str(train_roc)+'\\n'\\\n",
    "        +\"valid_roc\"+\":\"+str(valid_roc)+'\\n'\\\n",
    "        +\"train_roc_mean\"+\":\"+str(train_roc_mean)+'\\n'\\\n",
    "        +\"valid_roc_mean\"+\":\"+str(valid_roc_mean)+'\\n'\\\n",
    "        +\"train_prc_mean\"+\":\"+str(train_prc_mean)+'\\n'\\\n",
    "        +\"valid_prc_mean\"+\":\"+str(valid_prc_mean)+'\\n'\\\n",
    "        )\n",
    "    if (epoch - best_param[\"roc_epoch\"] >6) and (epoch - best_param[\"loss_epoch\"] >8):        \n",
    "        break\n",
    "        \n",
    "    train(model, train_df, optimizer, loss_function)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch:30\n",
      "test_roc:[0.7069322529822194, 0.9920634920634926, 0.9140589569160998, 0.9497372629655017, 0.8429051782111639, 0.8650194463509492, 0.5133470225872684, 0.7204450625869266, 0.8113887013279318, 0.6747440273037547, 0.8958617553433379, 0.7577747378020971, 0.9988615664845174, 0.8921435059037236, 0.9952120383036938, 0.8998643147896877, 0.8939809782608699]\n",
      "test_roc_mean: 0.8426082529519546\n",
      "test_prc_mean: 0.22154438821952206\n"
     ]
    }
   ],
   "source": [
    "# evaluate model\n",
    "best_model = torch.load('saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(best_param[\"roc_epoch\"])+'.pt')     \n",
    "\n",
    "# best_model_dict = best_model.state_dict()\n",
    "# best_model_wts = copy.deepcopy(best_model_dict)\n",
    "\n",
    "# model.load_state_dict(best_model_wts)\n",
    "# (best_model.align[0].weight == model.align[0].weight).all()\n",
    "test_roc, test_prc, test_losses = eval(best_model, test_df)\n",
    "\n",
    "print(\"best epoch:\"+str(best_param[\"roc_epoch\"])\n",
    "      +\"\\n\"+\"test_roc:\"+str(test_roc)\n",
    "      +\"\\n\"+\"test_roc_mean:\",str(np.array(test_roc).mean())\n",
    "      +\"\\n\"+\"test_prc_mean:\",str(np.array(test_prc).mean())\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
